# -*- coding: utf-8 -*-
# time: 2025/3/27 09:38
# file: embed_transformers.py
# author: hanson
"""
https://zhuanlan.zhihu.com/p/635670918 基于开源embedding模型的中文向量效果测试

"""
from transformers import AutoTokenizer, AutoModel
import torch

# 加载分词器和模型
#tokenizer = AutoTokenizer.from_pretrained("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
#model = AutoModel.from_pretrained("sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
# 假设你的本地模型路径是 './local-models/paraphrase-multilingual-MiniLM-L12-v2'
local_model_path=r"E:\soft\embedding\Ceceliachenen\paraphrase-multilingual-MiniLM-L12-v2"
# 加载分词器和模型
tokenizer = AutoTokenizer.from_pretrained(local_model_path)
model = AutoModel.from_pretrained(local_model_path)
# 示例文本
texts = ["Hello, how are you?", "How are you doing?", "Bonjour, comment ça va?"]

# 将文本转换为嵌入
inputs = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
print(inputs)
with torch.no_grad():
    embeddings = model(**inputs).last_hidden_state[:, 0, :]

# embeddings 现在包含了每个句子的嵌入表示
print(embeddings.shape)  # 输出: torch.Size([3, 384])